From 91334fd3b759d88be2a8e47b15dc12ace187edd5 Mon Sep 17 00:00:00 2001 From: Arnaud Bouchez Date: Tue, 3 Nov 2020 16:53:08 +0100 Subject: [PATCH] introducing low-level access to the Lizard/LZ5 API --- src/core/mormot.core.os.pas | 1 + src/lib/mormot.lib.lizard.pas | 572 ++++++++++++++++++++++++++++++++++ test/mormot2tests.dpr | 1 + test/mormot2tests.lpi | 6 +- 4 files changed, 579 insertions(+), 1 deletion(-) create mode 100644 src/lib/mormot.lib.lizard.pas diff --git a/src/core/mormot.core.os.pas b/src/core/mormot.core.os.pas index d336ef50f..a47debffa 100644 --- a/src/core/mormot.core.os.pas +++ b/src/core/mormot.core.os.pas @@ -1449,6 +1449,7 @@ TSynLibrary = class function GetProc(ProcName: PChar; Entry: PPointer; RaiseExceptionOnFailure: ExceptionClass = nil): boolean; /// cross-platform call to FreeLibrary() + set fHandle := 0 + // - as called by Destroy, but you can use it directly to reload the library procedure FreeLib; /// same as SafeLoadLibrary() but setting fLibraryPath and cwd on Windows function TryLoadLibrary(const aLibrary: array of TFileName; diff --git a/src/lib/mormot.lib.lizard.pas b/src/lib/mormot.lib.lizard.pas new file mode 100644 index 000000000..ca494314d --- /dev/null +++ b/src/lib/mormot.lib.lizard.pas @@ -0,0 +1,572 @@ +/// low-level access to the Lizard/LZ5 API +// - this unit is a part of the Open Source Synopse mORMot framework 2, +// licensed under a MPL/GPL/LGPL three license - see LICENSE.md +unit mormot.lib.lizard; + + +{ + ***************************************************************************** + + Cross-Platform and Cross-Compiler Lizard (LZ5) API + - Low-Level Lizard API Process + - TAlgoLizard/TAlgoLizardFast/TAlgoLizardHuffman High-Level Algorithms + + ***************************************************************************** + + + Some numbers, for a 53MB log file (taken from a production server): + FPC Win32 + TAlgoSynLz 53 MB->5 MB: comp 563.7 MB/s decomp 815.3 MB/s + TAlgoLizard 53 MB->3.9 MB: comp 54.6 MB/s decomp 1.1 GB/s + TAlgoLizardFast 53 MB->6.9 MB: comp 493.8 MB/s decomp 1 GB/s + TAlgoDeflate 53 MB->4.8 MB: comp 28.2 MB/s decomp 212 MB/s + TAlgoDeflateFast 53 MB->7 MB: comp 58.6 MB/s decomp 182.5 MB/s + FPC Win64 + TAlgoSynLz 53 MB->5 MB: comp 635.4 MB/s decomp 923.5 MB/s + TAlgoLizard 53 MB->3.9 MB: comp 61 MB/s decomp 1.8 GB/s + TAlgoLizardFast 53 MB->6.8 MB: comp 674.2 MB/s decomp 1.6 GB/s + TAlgoDeflate 53 MB->4.8 MB: comp 40.2 MB/s decomp 255.1 MB/s + TAlgoDeflateFast 53 MB->7 MB: comp 81.2 MB/s decomp 219.9 MB/s + FPC Linux32 + TAlgoSynLz 53 MB->5 MB: comp 533.4 MB/s decomp 472.5 MB/s + TAlgoLizard 53 MB->3.9 MB: comp 44.8 MB/s decomp 1.2 GB/s + TAlgoLizardFast 53 MB->6.9 MB: comp 515.5 MB/s decomp 1 GB/s + TAlgoDeflate 53 MB->4.8 MB: comp 60.2 MB/s decomp 413.3 MB/s + TAlgoDeflateFast 53 MB->7 MB: comp 121.8 MB/s decomp 336.7 MB/s + FPC Linux64 + TAlgoSynLz 53 MB->5 MB: comp 626.4 MB/s decomp 906.8 MB/s + TAlgoLizard 53 MB->3.9 MB: comp 53.6 MB/s decomp 1.8 GB/s + TAlgoLizardFast 53 MB->6.8 MB: comp 700.3 MB/s decomp 1.6 GB/s + TAlgoDeflate 53 MB->4.8 MB: comp 70.5 MB/s decomp 544.5 MB/s + TAlgoDeflateFast 53 MB->7 MB: comp 141.4 MB/s decomp 420.2 MB/s + Conclusion: SynLZ has the best compression ratio for its compression speed, + but Lizard is much faster at decompression, when working with big log filesiles. + For small files (= Lizard_compressBound(srcSize) + // - returns number of bytes written into dst (necessarily <= maxDstSize), + // or 0 if compression fails due to too small maxDstSize, <0 on other failure + // - compressionLevel is from LIZARD_MIN_CLEVEL (10) to LIZARD_MAX_CLEVEL(49), + // any value <10 (e.g. 0) will use 17, and value >49 will use 49 + // $ Lev Comp Decomp CompSize Ratio + // $ 7332 MB/s 8719 MB/s 211947520 100.00 (move) + // $ 10 346 MB/s 2610 MB/s 103402971 48.79 + // $ 12 103 MB/s 2458 MB/s 86232422 40.69 + // $ 15 50 MB/s 2552 MB/s 81187330 38.31 + // $ 19 3.04 MB/s 2497 MB/s 77416400 36.53 + // $ 21 157 MB/s 1795 MB/s 89239174 42.10 + // $ 23 30 MB/s 1778 MB/s 81097176 38.26 + // $ 26 6.63 MB/s 1734 MB/s 74503695 35.15 + // $ 29 1.37 MB/s 1634 MB/s 68694227 32.41 + // $ 30 246 MB/s 909 MB/s 85727429 40.45 + // $ 32 94 MB/s 1244 MB/s 76929454 36.30 + // $ 35 47 MB/s 1435 MB/s 73850400 34.84 + // $ 39 2.94 MB/s 1502 MB/s 69807522 32.94 + // $ 41 126 MB/s 961 MB/s 76100661 35.91 + // $ 43 28 MB/s 1101 MB/s 70955653 33.48 + // $ 46 6.25 MB/s 1073 MB/s 65413061 30.86 + // $ 49 1.27 MB/s 1064 MB/s 60679215 28.63 + compress: function(src, dst: pointer; + srcSize, maxDstSize, compressionLevel: integer): integer; cdecl; + /// how much memory must be allocated for compress_extState() + sizeofState: function(compressionLevel: integer): integer; cdecl; + /// compresses using an external pre-allocated state buffer + compress_extState: function(state: pointer; + src, dst: pointer; srcSize, maxDstSize, + compressionLevel: integer): integer; cdecl; + /// decompresses srcSize bytes from src into already allocated dst buffer + // - returns number of bytes written to dst (<= maxDstSize), or <=0 on failure + // - this function is protected against buffer overflow exploits + decompress_safe: function(src, dst: pointer; + srcSize, maxDstSize: integer): integer; cdecl; + /// partial decompression srcSize bytes from src into already allocated dst buffer + // - returns number of bytes written to dst (<= maxDstSize), or <=0 on failure + // - number can be 1 then + if aRaiseNoException then + exit + else + raise EAlgoCompress.CreateUTF8('% has unexpected versionNumber=%', + [fLibrary.LibraryPath, versionNumber]); + // register TAlgoLizard/TAlgoLizardFast/TAlgoLizardHuffman + inherited Create; + // if we reached here, the external library has been properly setup + fLoaded := true; + end + else if not aRaiseNoException then + raise EAlgoCompress.CreateUTF8('Unable to load % - %/'#13#10 + + 'Please download from https://synopse.info/files/SynLizardLibs.7z', + [aLibraryFile, SysErrorMessage(GetLastError)]); +end; + +destructor TSynLizardDynamic.Destroy; +begin + fLibrary.Free; + inherited; +end; + +class function TSynLizardDynamic.AlgoRegister: boolean; +var + lib: TSynLizardDynamic; +begin + result := Lizard <> nil; + if result then + // already registered (maybe as TSynLizardStatic) + exit; + lib := TSynLizardDynamic.Create('', true); + result := lib.Loaded; + if result then + Lizard := lib + else + lib.Free; +end; + +function TSynLizardDynamic.LibraryName: TFileName; +begin + result := fLibrary.LibraryPath; +end; + + +{ ****************** TAlgoLizard/TAlgoLizardFast/TAlgoLizardHuffman High-Level Algorithms } + +type + TAlgoLizard = class(TAlgoCompressWithNoDestLen) + protected + fCompressionLevel: integer; + function RawProcess(src, dst: pointer; srcLen, dstLen, dstMax: integer; + process: TAlgoCompressWithNoDestLenProcess): integer; override; + public + constructor Create; override; + function AlgoID: byte; override; + function AlgoCompressDestLen(PlainLen: integer): integer; override; + end; + + TAlgoLizardFast = class(TAlgoLizard) + public + constructor Create; override; + function AlgoID: byte; override; + end; + + TAlgoLizardHuffman = class(TAlgoLizard) + public + constructor Create; override; + function AlgoID: byte; override; + end; + + +{ TAlgoLizard } + +function TAlgoLizard.AlgoID: byte; +begin + result := 4; +end; + +constructor TAlgoLizard.Create; +begin + inherited Create; + fCompressionLevel := LIZARD_DEFAULT_CLEVEL; +end; + +function TAlgoLizard.AlgoCompressDestLen(PlainLen: integer): integer; +begin + if Lizard = nil then + result := 0 + else + result := Lizard.compressBound(PlainLen); +end; + +function TAlgoLizard.RawProcess(src, dst: pointer; srcLen, dstLen, dstMax: integer; + process: TAlgoCompressWithNoDestLenProcess): integer; +begin + if Lizard = nil then + result := 0 + else + case process of + doCompress: + result := Lizard.compress(src, dst, srcLen, dstLen, fCompressionLevel); + doUnCompress: + result := Lizard.decompress_safe(src, dst, srcLen, dstLen); + doUncompressPartial: + result := Lizard.decompress_safe_partial(src, dst, srcLen, dstLen, dstMax); + else + result := 0; + end; +end; + + +{ TAlgoLizardFast } + +constructor TAlgoLizardFast.Create; +begin + inherited Create; + fCompressionLevel := LIZARD_MIN_CLEVEL; +end; + +function TAlgoLizardFast.AlgoID: byte; +begin + result := 5; +end; + + +{ TAlgoLizardHuffman } + +constructor TAlgoLizardHuffman.Create; +begin + inherited Create; + fCompressionLevel := LIZARD_HUFFMAN_CLEVEL; +end; + +function TAlgoLizardHuffman.AlgoID: byte; +begin + result := 6; +end; + + +// this constructor uses TAlgoLizard* classes so is defined hereafter + +{ TSynLizard } + +constructor TSynLizard.Create; +begin + if AlgoLizard = nil then + AlgoLizard := TAlgoLizard.Create; + if AlgoLizardFast = nil then + AlgoLizardFast := TAlgoLizardFast.Create; + if AlgoLizardHuffman = nil then + AlgoLizardHuffman := TAlgoLizardHuffman.Create; +end; + + + + +initialization + {$ifndef LIZARD_EXTERNALONLY} + Lizard := TSynLizardStatic.Create; + {$endif LIZARD_EXTERNALONLY} + +finalization + Lizard.Free; + +end. + diff --git a/test/mormot2tests.dpr b/test/mormot2tests.dpr index cc9189701..6a7ef6bd1 100644 --- a/test/mormot2tests.dpr +++ b/test/mormot2tests.dpr @@ -35,6 +35,7 @@ uses mormot.core.interfaces in '..\src\core\mormot.core.interfaces.pas', mormot.core.zip in '..\src\core\mormot.core.zip.pas', mormot.lib.z in '..\src\lib\mormot.lib.z.pas', + mormot.lib.lizard in '..\src\lib\mormot.lib.lizard.pas', mormot.net.sock in '..\src\net\mormot.net.sock.pas', mormot.net.http in '..\src\net\mormot.net.http.pas', mormot.lib.winhttp in '..\src\lib\mormot.lib.winhttp.pas', diff --git a/test/mormot2tests.lpi b/test/mormot2tests.lpi index b66dd4137..553e5dc00 100644 --- a/test/mormot2tests.lpi +++ b/test/mormot2tests.lpi @@ -541,7 +541,7 @@ - + @@ -806,6 +806,10 @@ + + + +