Skip to content

Commit

Permalink
finish merge conflicts
Browse files Browse the repository at this point in the history 
Signed-off-by: Xavier Dupre <[email protected]>
  • Loading branch information
@xadupre
xadupre committed Oct 2, 2023
1 parent a77a522 commit 74bd8e9
Show file tree
Hide file tree
Showing 7 changed files with 257 additions and 149 deletions.
11 changes: 6 additions & 5 deletions onnxmltools/convert/lightgbm/_parse.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,10 @@ def __init__(self, booster):
self.operator_name = "LgbmRegressor"
else:
raise NotImplementedError(
'Unsupported LightGbm objective: %r.' % self.objective_)
"Unsupported LightGbm objective: %r." % self.objective_
)
try:
average_output = self.booster_.attr('average_output')
average_output = self.booster_.attr("average_output")
except AttributeError:
average_output = self.booster_.params.get("average_output", None)
if average_output:
Expand All @@ -58,9 +59,9 @@ def _generate_classes(booster):
num_class = booster["num_class"]
else:
try:
num_class = booster.attr('num_class')
num_class = booster.attr("num_class")
except AttributeError:
num_class = booster.params.get('num_class', None)
num_class = booster.params.get("num_class", None)
if num_class is None:
dp = booster.dump_model(num_iteration=1)
num_class = dp["num_class"]
Expand All @@ -73,7 +74,7 @@ def get_objective(self):
if hasattr(self, "objective_") and self.objective_ is not None:
return self.objective_
try:
objective = self.booster_.attr('objective')
objective = self.booster_.attr("objective")
except AttributeError:
objective = self.booster_.params.get("objective", None)
if objective is not None:
Expand Down
59 changes: 33 additions & 26 deletions onnxmltools/convert/lightgbm/operator_converters/LightGbm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,10 +315,7 @@ def dump_booster_model(
if getattr(self, "is_mock", False):
return self.dump_model(), None
from lightgbm.basic import _LIB, _safe_call
<<<<<<< HEAD
=======

>>>>>>> 79c34e377fe3a24d22eabac010e464de061d7adf
try:
# lightgbm >= 4.0
from lightgbm.basic import (
Expand All @@ -344,34 +341,39 @@ def dump_booster_model(
handle = self._handle
except AttributeError:
handle = self.handle
_safe_call(_LIB.LGBM_BoosterDumpModel(
handle,
ctypes.c_int(start_iteration),
ctypes.c_int(num_iteration),
ctypes.c_int(importance_type_int),
ctypes.c_int64(buffer_len),
ctypes.byref(tmp_out_len),
ptr_string_buffer))
_safe_call(
_LIB.LGBM_BoosterDumpModel(
handle,
ctypes.c_int(start_iteration),
ctypes.c_int(num_iteration),
ctypes.c_int(importance_type_int),
ctypes.c_int64(buffer_len),
ctypes.byref(tmp_out_len),
ptr_string_buffer,
)
)
actual_len = tmp_out_len.value
# if buffer length is not long enough, reallocate a buffer
if actual_len > buffer_len:
string_buffer = ctypes.create_string_buffer(actual_len)
ptr_string_buffer = ctypes.c_char_p(
*[ctypes.addressof(string_buffer)])
ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
try:
# lightgbm >= 4.0
handle = self._handle
except AttributeError:
# lightgbm < 4.0
handle = self.handle
_safe_call(_LIB.LGBM_BoosterDumpModel(
handle,
ctypes.c_int(start_iteration),
ctypes.c_int(num_iteration),
ctypes.c_int(importance_type_int),
ctypes.c_int64(actual_len),
ctypes.byref(tmp_out_len),
ptr_string_buffer))
_safe_call(
_LIB.LGBM_BoosterDumpModel(
handle,
ctypes.c_int(start_iteration),
ctypes.c_int(num_iteration),
ctypes.c_int(importance_type_int),
ctypes.c_int64(actual_len),
ctypes.byref(tmp_out_len),
ptr_string_buffer,
)
)

class Hook(json.JSONDecoder):
"""
Expand Down Expand Up @@ -431,11 +433,16 @@ def hook(self, obj):
if verbose >= 2:
print("[dump_booster_model] to_json")
info = {}
ret = json.loads(string_buffer.value.decode('utf-8'), cls=Hook,
info=info, n_trees=self.num_trees(), verbose=verbose)
ret['pandas_categorical'] = json.loads(
json.dumps(self.pandas_categorical,
default=jdwn))
ret = json.loads(
string_buffer.value.decode("utf-8"),
cls=Hook,
info=info,
n_trees=self.num_trees(),
verbose=verbose,
)
ret["pandas_categorical"] = json.loads(
json.dumps(self.pandas_categorical, default=jdwn)
)
if verbose >= 2:
print("[dump_booster_model] end.")
return ret, info
Expand Down
5 changes: 0 additions & 5 deletions onnxmltools/utils/tests_helper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,18 +200,13 @@ def dump_data_and_model(
continue
if isinstance(allow_failure, str):
raise NotImplementedError("allow_failure is deprecated.")
<<<<<<< HEAD
output = compare_backend(b, runtime_test, options=extract_options(basename),
context=context, verbose=verbose)
=======
output = compare_backend(
b,
runtime_test,
options=extract_options(basename),
context=context,
verbose=verbose,
)
>>>>>>> 79c34e377fe3a24d22eabac010e464de061d7adf
if output is not None:
dest = os.path.join(folder, basename + ".backend.{0}.pkl".format(b))
names.append(dest)
Expand Down
4 changes: 0 additions & 4 deletions onnxmltools/utils/utils_backend.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,6 @@ class OnnxRuntimeAssertionError(AssertionError):
"""
Expected failure.
"""
<<<<<<< HEAD
pass
=======
>>>>>>> 79c34e377fe3a24d22eabac010e464de061d7adf


def is_backend_enabled(backend):
Expand Down
121 changes: 90 additions & 31 deletions tests/hummingbirdml/test_LightGbmTreeEnsembleConverters_hummingbird.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,30 @@ def test_lightgbm_booster_classifier(self):
X = numpy.array(X, dtype=numpy.float32)
y = [0, 1, 0, 1]
data = lightgbm.Dataset(X, label=y)
model = lightgbm.train({'boosting_type': 'gbdt', 'objective': 'binary',
'n_estimators': 3, 'min_child_samples': 1, 'num_thread': 1},
data)
model_onnx, prefix = convert_model(model, 'tree-based classifier',
[('input', FloatTensorType([None, 2]))], without_onnx_ml=True,
target_opset=HUMMINGBIRD_TARGET_OPSET,
zipmap=False)
dump_data_and_model(X, model, model_onnx,
basename=prefix + "BoosterBin" + model.__class__.__name__)
model = lightgbm.train(
{
"boosting_type": "gbdt",
"objective": "binary",
"n_estimators": 3,
"min_child_samples": 1,
"num_thread": 1,
},
data,
)
model_onnx, prefix = convert_model(
model,
"tree-based classifier",
[("input", FloatTensorType([None, 2]))],
without_onnx_ml=True,
target_opset=HUMMINGBIRD_TARGET_OPSET,
zipmap=False,
)
dump_data_and_model(
X,
model,
model_onnx,
basename=prefix + "BoosterBin" + model.__class__.__name__,
)

@unittest.skipIf(not hummingbird_installed(), reason="Hummingbird is not installed")
def test_lightgbm_booster_classifier_zipmap(self):
Expand Down Expand Up @@ -81,27 +96,55 @@ def test_lightgbm_booster_classifier_zipmap(self):
target_opset=HUMMINGBIRD_TARGET_OPSET,
)

model_onnx, prefix = convert_model(model, 'tree-based classifier',
[('input', FloatTensorType([None, 2]))], without_onnx_ml=True,
target_opset=HUMMINGBIRD_TARGET_OPSET, zipmap=False)
dump_data_and_model(X, model, model_onnx,
basename=prefix + "BoosterBin" + model.__class__.__name__)
model_onnx, prefix = convert_model(
model,
"tree-based classifier",
[("input", FloatTensorType([None, 2]))],
without_onnx_ml=True,
target_opset=HUMMINGBIRD_TARGET_OPSET,
zipmap=False,
)
dump_data_and_model(
X,
model,
model_onnx,
basename=prefix + "BoosterBin" + model.__class__.__name__,
)

@unittest.skipIf(not hummingbird_installed(), reason="Hummingbird is not installed")
def test_lightgbm_booster_multi_classifier(self):
X = [[0, 1], [1, 1], [2, 0], [1, 2], [-1, 2], [1, -2]]
X = numpy.array(X, dtype=numpy.float32)
y = [0, 1, 0, 1, 2, 2]
data = lightgbm.Dataset(X, label=y)
model = lightgbm.train({'boosting_type': 'gbdt', 'objective': 'multiclass',
'n_estimators': 3, 'min_child_samples': 1, 'num_class': 3, 'num_thread': 1},
data)
model_onnx, prefix = convert_model(model, 'tree-based classifier',
[('input', FloatTensorType([None, 2]))], without_onnx_ml=True,
target_opset=HUMMINGBIRD_TARGET_OPSET, zipmap=False)
dump_data_and_model(X, model, model_onnx,
basename=prefix + "BoosterBin" + model.__class__.__name__)
sess = InferenceSession(model_onnx.SerializeToString(), providers=["CPUExecutionProvider"])
model = lightgbm.train(
{
"boosting_type": "gbdt",
"objective": "multiclass",
"n_estimators": 3,
"min_child_samples": 1,
"num_class": 3,
"num_thread": 1,
},
data,
)
model_onnx, prefix = convert_model(
model,
"tree-based classifier",
[("input", FloatTensorType([None, 2]))],
without_onnx_ml=True,
target_opset=HUMMINGBIRD_TARGET_OPSET,
zipmap=False,
)
dump_data_and_model(
X,
model,
model_onnx,
basename=prefix + "BoosterBin" + model.__class__.__name__,
)
sess = InferenceSession(
model_onnx.SerializeToString(), providers=["CPUExecutionProvider"]
)
out = sess.get_outputs()
names = [o.name for o in out]
assert names == ["label", "probabilities"]
Expand All @@ -112,14 +155,30 @@ def test_lightgbm_booster_regressor(self):
X = numpy.array(X, dtype=numpy.float32)
y = [0, 1, 1.1]
data = lightgbm.Dataset(X, label=y)
model = lightgbm.train({'boosting_type': 'gbdt', 'objective': 'regression',
'n_estimators': 3, 'min_child_samples': 1, 'max_depth': 1},
data)
model_onnx, prefix = convert_model(model, 'tree-based binary regressor',
[('input', FloatTensorType([None, 2]))], without_onnx_ml=True,
target_opset=HUMMINGBIRD_TARGET_OPSET, zipmap=False)
dump_data_and_model(X, model, model_onnx,
basename=prefix + "BoosterBin" + model.__class__.__name__)
model = lightgbm.train(
{
"boosting_type": "gbdt",
"objective": "regression",
"n_estimators": 3,
"min_child_samples": 1,
"max_depth": 1,
},
data,
)
model_onnx, prefix = convert_model(
model,
"tree-based binary regressor",
[("input", FloatTensorType([None, 2]))],
without_onnx_ml=True,
target_opset=HUMMINGBIRD_TARGET_OPSET,
zipmap=False,
)
dump_data_and_model(
X,
model,
model_onnx,
basename=prefix + "BoosterBin" + model.__class__.__name__,
)

# Base test implementation comparing ONNXML and ONNX models.
def _test_lgbm(self, X, model, extra_config={}):
Expand Down
Loading

0 comments on commit 74bd8e9

Please sign in to comment.