From 5d86aa180959cfc8fcb025f397f2df34acb167c3 Mon Sep 17 00:00:00 2001 From: sisong Date: Mon, 6 Nov 2023 14:30:35 +0800 Subject: [PATCH 1/9] support long path on Windows OS; update HDiffPatch; --- HDiffPatch | 2 +- builds/vc/ApkNormalized.vcxproj | 12 ++++++++++++ builds/vc/ZipDiff.vcxproj | 12 ++++++++++++ builds/vc/ZipPatch.vcxproj | 12 ++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/HDiffPatch b/HDiffPatch index a970d36..ef3bf6a 160000 --- a/HDiffPatch +++ b/HDiffPatch @@ -1 +1 @@ -Subproject commit a970d3655fe86cae03ea1d05e9258132e31d54f7 +Subproject commit ef3bf6a60585c8e398c654305e395a36d8aaf92e diff --git a/builds/vc/ApkNormalized.vcxproj b/builds/vc/ApkNormalized.vcxproj index bc006a2..900023c 100644 --- a/builds/vc/ApkNormalized.vcxproj +++ b/builds/vc/ApkNormalized.vcxproj @@ -121,6 +121,9 @@ true + + ..\..\HDiffPatch\builds\vc\longPathAware.exe.manifest %(AdditionalManifestFiles) + @@ -133,6 +136,9 @@ true + + ..\..\HDiffPatch\builds\vc\longPathAware.exe.manifest %(AdditionalManifestFiles) + @@ -151,6 +157,9 @@ false + + ..\..\HDiffPatch\builds\vc\longPathAware.exe.manifest %(AdditionalManifestFiles) + @@ -170,6 +179,9 @@ false + + ..\..\HDiffPatch\builds\vc\longPathAware.exe.manifest %(AdditionalManifestFiles) + diff --git a/builds/vc/ZipDiff.vcxproj b/builds/vc/ZipDiff.vcxproj index a5e4220..beae942 100644 --- a/builds/vc/ZipDiff.vcxproj +++ b/builds/vc/ZipDiff.vcxproj @@ -141,6 +141,9 @@ true + + ..\..\HDiffPatch\builds\vc\longPathAware.exe.manifest %(AdditionalManifestFiles) + @@ -153,6 +156,9 @@ true + + ..\..\HDiffPatch\builds\vc\longPathAware.exe.manifest %(AdditionalManifestFiles) + @@ -171,6 +177,9 @@ false + + ..\..\HDiffPatch\builds\vc\longPathAware.exe.manifest %(AdditionalManifestFiles) + @@ -190,6 +199,9 @@ false + + ..\..\HDiffPatch\builds\vc\longPathAware.exe.manifest %(AdditionalManifestFiles) + diff --git a/builds/vc/ZipPatch.vcxproj b/builds/vc/ZipPatch.vcxproj index 20e66b2..7f3023e 100644 --- a/builds/vc/ZipPatch.vcxproj +++ b/builds/vc/ZipPatch.vcxproj @@ -126,6 +126,9 @@ true + + ..\..\HDiffPatch\builds\vc\longPathAware.exe.manifest %(AdditionalManifestFiles) + @@ -138,6 +141,9 @@ true + + ..\..\HDiffPatch\builds\vc\longPathAware.exe.manifest %(AdditionalManifestFiles) + @@ -156,6 +162,9 @@ false + + ..\..\HDiffPatch\builds\vc\longPathAware.exe.manifest %(AdditionalManifestFiles) + @@ -175,6 +184,9 @@ false + + ..\..\HDiffPatch\builds\vc\longPathAware.exe.manifest %(AdditionalManifestFiles) + From ef6306ef07cbbdb881f8e0678329f52bd671937e Mon Sep 17 00:00:00 2001 From: sisong Date: Mon, 4 Dec 2023 09:01:17 +0800 Subject: [PATCH 2/9] update libs & descriptions; --- HDiffPatch | 2 +- README.md | 7 +++---- lzma | 2 +- src/diff/Differ.cpp | 2 +- src/zip_diff.cpp | 2 +- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/HDiffPatch b/HDiffPatch index ef3bf6a..e2d2052 160000 --- a/HDiffPatch +++ b/HDiffPatch @@ -1 +1 @@ -Subproject commit ef3bf6a60585c8e398c654305e395a36d8aaf92e +Subproject commit e2d205200b5dc798880f373c79cbd01d7319f969 diff --git a/README.md b/README.md index b1a2530..981da76 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,6 @@ You can use this library (and Android NDK) to delta update your Apk. [zlib]: https://github.com/madler/zlib [lzma]: https://www.7-zip.org/sdk.html [Apk v2 sign]: https://source.android.com/security/apksigning/v2 -[Apk v3 sign]: https://source.android.com/security/apksigning/v3 [Jar sign]: https://docs.oracle.com/javase/8/docs/technotes/guides/jar/jar.html#Signed_JAR_File [BsDiff]: http://www.daemonology.net/bsdiff/ [archive-patcher]: https://github.com/google/archive-patcher @@ -40,11 +39,11 @@ ZipPatch() support multi-thread parallel compress mode when writing zip file, wh * NOTE: if your need newZip(patch result) file byte by byte equal, `Released newZip` := **ApkNormalized**(newZip) before run ZipDiff, AND You should not modify the zlib version (unless it is certified compatible); if your apk(or jar) file used [Jar sign](Apk v1 sign), is same as zip file; -if your apk used [Apk v2 sign](or [Apk v3 sign]), `Released newZip` := AndroidSDK#apksigner(**ApkNormalized**(AndroidSDK#apksigner(newZip))) before ZipDiff; +if your apk used [Apk v2 sign](or [later](https://source.android.com/security/apksigning/v3)), `Released newApk` := AndroidSDK#apksigner(**ApkNormalized**(newApk)) before ZipDiff; * NOTE: -ApkDiffPath can't be used in the Android app store, because it requires re-signing apk; -[sfpatcher] like [archive-patcher], is designed for Android app store, but patch is much faster than archive-patcher. +ApkDiffPath can't be used by Android app store, because it requires re-signing apks before diff. +[sfpatcher] not require re-signing apks (like [archive-patcher]), is designed for Android app store, patch speed up by a factor of xx than archive-patcher & run with O(1) memory. --- ## Why [ApkDiffPatch] diff --git a/lzma b/lzma index 3e430b4..e2ff7f0 160000 --- a/lzma +++ b/lzma @@ -1 +1 @@ -Subproject commit 3e430b4637b9f20720d979a8b1680c075e83532e +Subproject commit e2ff7f0c42d722bad95cce4a26966eaf8685487d diff --git a/src/diff/Differ.cpp b/src/diff/Differ.cpp index 949599c..c304e57 100644 --- a/src/diff/Differ.cpp +++ b/src/diff/Differ.cpp @@ -304,7 +304,7 @@ static bool checkZipInfo(UnZipper* oldZip,UnZipper* newZip){ if (newIsV2Sign&(!newZip->_isDataNormalized)){ //maybe bring apk can't install ERROR! printf(" ERROR: newZip not Normalized, need do " - "newZip=AndroidSDK#apksigner(ApkNormalized(AndroidSDK#apksigner(newZip))) before running ZipDiff!\n"); + "newZip=AndroidSDK#apksigner(ApkNormalized(newZip)) before running ZipDiff!\n"); isOk=false; } if ((!newIsV2Sign)&&UnZipper_isHaveApkV2orV3SignTag_in_ApkV1SignFile(newZip)){ diff --git a/src/zip_diff.cpp b/src/zip_diff.cpp index bf8a050..3c97354 100644 --- a/src/zip_diff.cpp +++ b/src/zip_diff.cpp @@ -390,7 +390,7 @@ int zipdiff_cmd_line(int argc, const char * argv[]) { } break; case CHECK_SAME_LIKE_TRUE__BYTE_BY_BYTE_EQUAL_ERROR:{ printf(" check ZipPatch result Byte By Byte Equal ERROR!\n"); - printf(" (did newZip=AndroidSDK#apksigner(ApkNormalized(AndroidSDK#apksigner(newZip))) before running ZipDiff?)\n"); + printf(" (did newZip=AndroidSDK#apksigner(ApkNormalized(newZip)) before running ZipDiff?)\n"); } break; case CHECK_SAME_LIKE_ERROR:{ printf(" check ZipPatch result zip data ERROR!\n"); From d393005da63f4a6b1d41e8b62c9b7b48f053b5e1 Mon Sep 17 00:00:00 2001 From: sisong Date: Thu, 7 Dec 2023 10:14:58 +0800 Subject: [PATCH 3/9] removeNonEmptyDirs(); other name recode; --- src/apk_normalized.cpp | 6 +-- src/diff/DiffData.cpp | 17 ++++---- src/diff/DiffData.h | 4 +- src/normalized/normalized.cpp | 74 ++++++++++++++++++++++++++++++----- src/normalized/normalized.h | 2 +- 5 files changed, 79 insertions(+), 24 deletions(-) diff --git a/src/apk_normalized.cpp b/src/apk_normalized.cpp index e6b47ac..0d22ac8 100644 --- a/src/apk_normalized.cpp +++ b/src/apk_normalized.cpp @@ -172,16 +172,16 @@ int normalized_cmd_line(int argc, const char * argv[]){ const char* dstApk=arg_values[1]; printf("src: \"%s\"\nout: \"%s\"\n",srcApk,dstApk); double time0=clock_s(); - int apkV1SignFilesRemoved=0; + int apkFilesRemoved=0; if (!ZipNormalized(srcApk,dstApk,(int)alignSize,(int)compressLevel, - isNotCompressEmptyFile?true:false,&apkV1SignFilesRemoved)){ + isNotCompressEmptyFile?true:false,&apkFilesRemoved)){ printf("\nrun ApkNormalized ERROR!\n"); return 1; } printf("run ApkNormalized ok!\n"); //check - if (!getZipIsSame(srcApk,dstApk,apkV1SignFilesRemoved)){ + if (!getZipIsSame(srcApk,dstApk,apkFilesRemoved)){ printf("ApkNormalized result file check ERROR!\n"); return 1; } diff --git a/src/diff/DiffData.cpp b/src/diff/DiffData.cpp index a6c17c6..2e719bc 100644 --- a/src/diff/DiffData.cpp +++ b/src/diff/DiffData.cpp @@ -63,7 +63,7 @@ bool zipFileData_isSame(UnZipper* selfZip,int selfIndex,UnZipper* srcZip,int src bool getZipIsSameWithStream(const hpatch_TStreamInput* oldZipStream, const hpatch_TStreamInput* newZipStream, - int newApkV1SignFilesRemoved,bool* out_isOldHaveApkV2Sign){ + int newApkFilesRemoved,bool* out_isOldHaveApkV2Sign){ UnZipper newZip; UnZipper oldZip; std::map oldMap; @@ -81,12 +81,12 @@ bool getZipIsSameWithStream(const hpatch_TStreamInput* oldZipStream, } fileCount=UnZipper_fileCount(&oldZip); - test_clear(fileCount==UnZipper_fileCount(&newZip)+newApkV1SignFilesRemoved); + test_clear(fileCount==UnZipper_fileCount(&newZip)+newApkFilesRemoved); for (int i=0;i::iterator it=oldMap.find(zipFile_name(&newZip,i)); test_clear(it!=oldMap.end()); int old_i=it->second; @@ -94,11 +94,12 @@ bool getZipIsSameWithStream(const hpatch_TStreamInput* oldZipStream, test_clear(UnZipper_file_crc32(&oldZip,old_i)==UnZipper_file_crc32(&newZip,i)); test_clear(zipFileData_isSame(&oldZip,old_i,&newZip,i)); } - check_clear(oldMap.size()==newApkV1SignFilesRemoved); - if (newApkV1SignFilesRemoved>0){ + check_clear(oldMap.size()==newApkFilesRemoved); + if (newApkFilesRemoved>0){ for (std::map::iterator it=oldMap.begin();it!=oldMap.end();++it){ int old_i=it->second; - check_clear(UnZipper_file_isApkV1Sign(&oldZip,old_i)); + check_clear(UnZipper_file_isApkV1Sign(&oldZip,old_i) + ||(UnZipper_file_uncompressedSize(&oldZip,old_i)==0)); } } @@ -109,7 +110,7 @@ bool getZipIsSameWithStream(const hpatch_TStreamInput* oldZipStream, return result;} bool getZipIsSame(const char* oldZipPath,const char* newZipPath, - int newApkV1SignFilesRemoved,bool* out_isOldHaveApkV2Sign){ + int newApkFilesRemoved,bool* out_isOldHaveApkV2Sign){ hpatch_TFileStreamInput oldZipStream; hpatch_TFileStreamInput newZipStream; bool result=true; @@ -120,7 +121,7 @@ bool getZipIsSame(const char* oldZipPath,const char* newZipPath, check_clear(hpatch_TFileStreamInput_open(&oldZipStream,oldZipPath)); check_clear(hpatch_TFileStreamInput_open(&newZipStream,newZipPath)); result=getZipIsSameWithStream(&oldZipStream.base,&newZipStream.base, - newApkV1SignFilesRemoved,out_isOldHaveApkV2Sign); + newApkFilesRemoved,out_isOldHaveApkV2Sign); clear: _isInClear=true; check_clear(hpatch_TFileStreamInput_close(&newZipStream)); diff --git a/src/diff/DiffData.h b/src/diff/DiffData.h index 1727e67..563f2db 100644 --- a/src/diff/DiffData.h +++ b/src/diff/DiffData.h @@ -46,10 +46,10 @@ bool zipFileData_isSame(UnZipper* self,int selfIndex,UnZipper* srcZip,int srcIndex);//byte by byte test bool getZipIsSame(const char* oldZipPath,const char* newZipPath, - int newApkV1SignFilesRemoved=0,bool* out_isOldHaveApkV2Sign=0); + int newApkFilesRemoved=0,bool* out_isOldHaveApkV2Sign=0); bool getZipIsSameWithStream(const hpatch_TStreamInput* oldZipStream, const hpatch_TStreamInput* newZipStream, - int newApkV1SignFilesRemoved=0,bool* out_isOldHaveApkV2Sign=0); + int newApkFilesRemoved=0,bool* out_isOldHaveApkV2Sign=0); bool getCompressedIsNormalized(UnZipper* zip,int* out_zlibCompressLevel, int* out_zlibCompressMemLevel,bool testReCompressedByApkV2Sign=false); //只检查压缩数据是否标准化; bool getCompressedIsNormalizedBy(UnZipper* zip,int zlibCompressLevel, diff --git a/src/normalized/normalized.cpp b/src/normalized/normalized.cpp index 4f32289..626666f 100644 --- a/src/normalized/normalized.cpp +++ b/src/normalized/normalized.cpp @@ -42,14 +42,27 @@ struct TFileValue{ static bool inline isInSignDir(const std::string& s){ return (s.find("META-INF/")==0)||(s.find("META-INF\\")==0); } + static bool inline isDirTag(char c){ + return (c=='\\')|(c=='/'); + } static bool inline isEndWith(const std::string& s,const char* sub){ return (s.find(sub)==s.size()-strlen(sub)); } + static bool inline isNameWith(const std::string& s,const char* name){ + if (!isEndWith(s,name)) return false; + size_t nameL=s.size()-strlen(name)-1; + if (s.size()==nameL) return true; + size_t s1=s.size()-nameL-1; + return isDirTag(s[s1]); + } static bool inline isSignMFFile(const std::string& s){ return (s=="META-INF/MANIFEST.MF")||(s=="META-INF\\MANIFEST.MF"); } + static bool inline isCertFile(const std::string& s){ + return isNameWith(s,"stamp-cert-sha256"); + } struct TCmp{ - inline TCmp(int fileCount):_fileCount(fileCount){} + inline explicit TCmp(int fileCount):_fileCount(fileCount){} size_t _v(const TFileValue& x)const{ size_t xi=x.fileIndex; if (isInSignDir(x.fileName)){ @@ -65,25 +78,61 @@ struct TFileValue{ } int _fileCount; }; + struct TCmpByName{ + inline bool operator ()(const TFileValue* x,const TFileValue* y)const{ + return x->fileNamefileName; + } + }; }; -static void getFiles(const UnZipper* zip,std::vector& out_files){ +static void getAllFiles(const UnZipper* zip,std::vector& out_files){ int fileCount=UnZipper_fileCount(zip); + out_files.resize(fileCount); for (int i=0; i& files){ + int fileCount=UnZipper_fileCount(zip); + { + std::vector rfiles(fileCount); + for (int i=0; ifileIndex)>0) continue; + const std::string& nx=rfiles[i]->fileName; + const std::string& ny=rfiles[i+1]->fileName; + const size_t nL=nx.size(); + if (nL>ny.size()) continue; + if (0!=memcmp(nx.c_str(),ny.c_str(),nL)) continue; + if (((nL>0)&&TFileValue::isDirTag(nx[nL-1]))|| + ((nLfileIndex=-1; //need remove + } + } + int insert=0; + for (int i=0; i files; - getFiles(&unzipper,files); + getAllFiles(&unzipper,files); + apkFilesRemoved=removeNonEmptyDirs(&unzipper,files); + fileCount=(int)files.size(); std::sort(files.begin(),files.end(),TFileValue::TCmp(fileCount)); for (int i=0; i0){ if (isHaveApkV2Sign){ printf("WARNING: src removed JarSign(ApkV1Sign) (%d file, need re sign)\n",jarSignFileCount); - for (size_t i=0;i Date: Fri, 8 Dec 2023 13:54:03 +0800 Subject: [PATCH 4/9] cmdline support "-ap-isPageAlignSoFile"; ( if found uncompressed .so file in the zip, need align it to 4k page? default 1); --- builds/vc/ApkNormalized.vcxproj | 8 ++--- src/apk_normalized.cpp | 40 ++++++++++++++++-------- src/normalized/normalized.cpp | 11 ++++--- src/normalized/normalized.h | 5 ++- src/patch/Zipper.cpp | 55 +++++++++++++++++++++++---------- src/patch/Zipper.h | 7 +++-- 6 files changed, 83 insertions(+), 43 deletions(-) diff --git a/builds/vc/ApkNormalized.vcxproj b/builds/vc/ApkNormalized.vcxproj index 900023c..0079098 100644 --- a/builds/vc/ApkNormalized.vcxproj +++ b/builds/vc/ApkNormalized.vcxproj @@ -114,7 +114,7 @@ Level3 Disabled - WIN32;_DEBUG;_IS_USED_MULTITHREAD=0;%(PreprocessorDefinitions) + WIN32;_CRT_SECURE_NO_WARNINGS;_DEBUG;_IS_USED_MULTITHREAD=0;%(PreprocessorDefinitions) Default /wd4819 %(AdditionalOptions) @@ -129,7 +129,7 @@ Level3 Disabled - WIN32;_DEBUG;_IS_USED_MULTITHREAD=0;%(PreprocessorDefinitions) + WIN32;_CRT_SECURE_NO_WARNINGS;_DEBUG;_IS_USED_MULTITHREAD=0;%(PreprocessorDefinitions) Default /wd4819 %(AdditionalOptions) @@ -143,7 +143,7 @@ Level3 - WIN32;NDEBUG;_IS_USED_MULTITHREAD=0;%(PreprocessorDefinitions) + WIN32;_CRT_SECURE_NO_WARNINGS;NDEBUG;_IS_USED_MULTITHREAD=0;%(PreprocessorDefinitions) Default Speed true @@ -164,7 +164,7 @@ Level3 - WIN32;NDEBUG;_IS_USED_MULTITHREAD=0;%(PreprocessorDefinitions) + WIN32;_CRT_SECURE_NO_WARNINGS;NDEBUG;_IS_USED_MULTITHREAD=0;%(PreprocessorDefinitions) Default Speed true diff --git a/src/apk_normalized.cpp b/src/apk_normalized.cpp index 0d22ac8..11e1062 100644 --- a/src/apk_normalized.cpp +++ b/src/apk_normalized.cpp @@ -43,7 +43,7 @@ #endif static void printUsage(){ - printf("usage: ApkNormalized srcApk out_newApk [-nce-isNotCompressEmptyFile] [-cl-compressLevel] [-as-alignSize] [-v] \n" + printf("usage: ApkNormalized srcApk normalizedApk [-cl-compressLevel]\n" "options:\n" " input srcApk file can *.zip *.jar *.apk file type;\n" " ApkNormalized normalized zip file:\n" @@ -52,19 +52,24 @@ static void printUsage(){ " remove all data descriptor, reserve & normalized Extra field and Comment,\n" " compatible with jar sign(apk v1 sign), etc...\n" " if apk file used apk v2 sign, must re sign apk file after ApkNormalized;\n" - " release newApk:=AndroidSDK#apksigner(out_newApk)\n" - " -nce-isNotCompressEmptyFile\n" - " if found compressed empty file in the zip, need change it to not compressed?\n" - " -nce-0 keep the original compress setting for empty file;\n" - " WARNING: if have compressed empty file,\n" - " it can't patch by old ZipPatch(version6) + printf("WARNING: not recommended 7,8,9, compress rate is high, but compress speed is very slow when patching.\n"); + _options_check(arg_values_size==2,"count"); const char* srcApk=arg_values[0]; const char* dstApk=arg_values[1]; @@ -174,7 +188,7 @@ int normalized_cmd_line(int argc, const char * argv[]){ double time0=clock_s(); int apkFilesRemoved=0; if (!ZipNormalized(srcApk,dstApk,(int)alignSize,(int)compressLevel, - isNotCompressEmptyFile?true:false,&apkFilesRemoved)){ + (bool)isNotCompressEmptyFile,(bool)isPageAlignSoFile,&apkFilesRemoved)){ printf("\nrun ApkNormalized ERROR!\n"); return 1; } diff --git a/src/normalized/normalized.cpp b/src/normalized/normalized.cpp index 626666f..558979a 100644 --- a/src/normalized/normalized.cpp +++ b/src/normalized/normalized.cpp @@ -36,6 +36,8 @@ if (!(value)){ printf(#value" ERROR!\n"); \ result=false; if (!_isInClear){ goto clear; } } } +#define kSoPageAlignSize (1024*4) + struct TFileValue{ std::string fileName; int fileIndex; @@ -131,8 +133,8 @@ inline static bool isCompressedEmptyFile(const UnZipper* unzipper,int fileIndex) &&UnZipper_file_isCompressed(unzipper,fileIndex); } -bool ZipNormalized(const char* srcApk,const char* dstApk, - int ZipAlignSize,int compressLevel,bool isNotCompressEmptyFile,int* out_apkFilesRemoved){ +bool ZipNormalized(const char* srcApk,const char* dstApk,int ZipAlignSize,int compressLevel, + bool isNotCompressEmptyFile,bool isPageAlignSoFile,int* out_apkFilesRemoved){ bool result=true; bool _isInClear=false; int fileCount=0; @@ -148,7 +150,8 @@ bool ZipNormalized(const char* srcApk,const char* dstApk, check(UnZipper_openFile(&unzipper,srcApk)); fileCount=UnZipper_fileCount(&unzipper); - check(Zipper_openFile(&zipper,dstApk,fileCount,ZipAlignSize,compressLevel,kDefaultZlibCompressMemLevel)); + check(Zipper_openFile(&zipper,dstApk,fileCount,ZipAlignSize,isPageAlignSoFile?kSoPageAlignSize:ZipAlignSize, + compressLevel,kDefaultZlibCompressMemLevel)); isHaveApkV2Sign=UnZipper_isHaveApkV2Sign(&unzipper); isHaveApkV3Sign=UnZipper_isHaveApkV3Sign(&unzipper); { @@ -187,7 +190,7 @@ bool ZipNormalized(const char* srcApk,const char* dstApk, check(Zipper_file_append_set_new_isCompress(&zipper,false)); printf("NOTE: \"%s\" is a compressed empty file, change to uncompressed!\n",fileName.c_str()); }else{ - printf("WARNING: \"%s\" is a compressed empty file, can't patch by old ZipPatch(version_threadNum>1) bool Zipper_openStream(Zipper* self,const hpatch_TStreamOutput* zipStream,int fileEntryMaxCount, - int ZipAlignSize,int compressLevel,int compressMemLevel){ + int ZipAlignSize,int SoPageAlignSize,int compressLevel,int compressMemLevel){ #if (_IS_NEED_FIXED_ZLIB_VERSION) check(0==strcmp(kNormalizedZlibVersion,zlibVersion()));//fixed zlib version #endif assert(ZipAlignSize>0); + assert(SoPageAlignSize>0); if (ZipAlignSize<=0) ZipAlignSize=1; + if (SoPageAlignSize<=0) SoPageAlignSize=1; checkCompressSet(compressLevel,compressMemLevel); self->_ZipAlignSize=ZipAlignSize; + self->_SoPageAlignSize=SoPageAlignSize; self->_compressLevel=compressLevel; self->_compressMemLevel=compressMemLevel; @@ -900,12 +911,12 @@ void Zipper_by_multi_thread(Zipper* self,int threadNum){ } bool Zipper_openFile(Zipper* self,const char* zipFileName,int fileEntryMaxCount, - int ZipAlignSize,int compressLevel,int compressMemLevel){ + int ZipAlignSize,int SoPageAlignSize,int compressLevel,int compressMemLevel){ assert(self->_fileStream.m_file==0); check(hpatch_TFileStreamOutput_open(&self->_fileStream,zipFileName,(hpatch_StreamPos_t)(-1))); hpatch_TFileStreamOutput_setRandomOut(&self->_fileStream,hpatch_TRUE); bool result=Zipper_openStream(self,&self->_fileStream.base,fileEntryMaxCount, - ZipAlignSize,compressLevel,compressMemLevel); + ZipAlignSize,SoPageAlignSize,compressLevel,compressMemLevel); if (!result){ hpatch_TFileStreamOutput_close(&self->_fileStream); self->_fileStream.m_file=0; @@ -960,8 +971,7 @@ inline static bool _writeUInt16(Zipper* self,uint16_t v){ } -inline static size_t _getAlignSkipLen(const Zipper* self,size_t curPos){ - size_t align=self->_ZipAlignSize; +inline static size_t _getAlignSkipLen(size_t curPos,size_t align){ return align-1-(curPos+align-1)%align; } inline static bool _writeAlignSkip(Zipper* self,size_t alignSkipLen){ @@ -977,7 +987,7 @@ inline static bool _writeAlignSkip(Zipper* self,size_t alignSkipLen){ return true; } -#define assert_align(self) assert((self->_curFilePos%self->_ZipAlignSize)==0) +#define assert_align(curPos,alignSize) assert(((curPos)%(alignSize))==0) static bool _write_fileHeaderInfo(Zipper* self,int fileIndex,UnZipper* srcZip,int srcFileIndex,bool isFullInfo){ const TByte* headBuf=fileHeaderBuf(srcZip,srcFileIndex); @@ -995,19 +1005,29 @@ static bool _write_fileHeaderInfo(Zipper* self,int fileIndex,UnZipper* srcZip,in uint16_t extraFieldLen=readUInt16(headBuf+30); uint16_t extraFieldLen_for_align=extraFieldLen; bool isNeedAlign=(!isFullInfo)&&(!isCompressed); //dir or 0 size file need align too, same AndroidSDK#zipalign + bool isNeedSoAlign=false; if (isNeedAlign){ size_t headInfoLen=30+fileNameLen+extraFieldLen; - size_t skipLen=_getAlignSkipLen(self,self->_curFilePos+headInfoLen); + size_t skipLen=_getAlignSkipLen(self->_curFilePos+headInfoLen,self->_ZipAlignSize); + if (UnZipper_file_is_nameEndWith(srcZip,srcFileIndex,".so")){ + size_t skipSoLen=_getAlignSkipLen(self->_curFilePos+headInfoLen,self->_SoPageAlignSize); + if (skipSoLen!=skipLen){ + skipLen=skipSoLen; + isNeedSoAlign=true; + } + } if (skipLen>0){ - if (fileIndex==0){//只能扩展第一个entry的extraField来对齐; + if ((isNeedSoAlign) //为了保持兼容旧的patch,用extraField来对齐; + ||(fileIndex==0)){//只能扩展第一个entry的extraField来对齐; extraFieldLen_for_align+=(uint16_t)skipLen; - //WARNING - char fileName[hpatch_kPathMaxSize]; - if (fileNameLen+1>hpatch_kPathMaxSize) return false; - memcpy(fileName,UnZipper_file_nameBegin(srcZip,srcFileIndex),fileNameLen); - fileName[fileNameLen]='\0'; - printf("WARNING: \""); hpatch_printPath_utf8(fileName); - printf("\" file's extraField adding %d byte 0 for align!\n",(int)skipLen); + {// print WARNING + char fileName[hpatch_kPathMaxSize]; + if (fileNameLen+1>hpatch_kPathMaxSize) return false; + memcpy(fileName,UnZipper_file_nameBegin(srcZip,srcFileIndex),fileNameLen); + fileName[fileNameLen]='\0'; + printf("WARNING: \""); hpatch_printPath_utf8(fileName); + printf("\" file's extraField adding %d byte 0 for align!\n",(int)skipLen); + } }else{ check(_writeAlignSkip(self,skipLen));//利用entry之间的空间对齐; } @@ -1044,7 +1064,8 @@ static bool _write_fileHeaderInfo(Zipper* self,int fileIndex,UnZipper* srcZip,in check(_writeAlignSkip(self,extraFieldLen_for_align-extraFieldLen)); if (fileCommentLen>0) check(_write(self,headBuf+46+fileNameLen+extraFieldLen,fileCommentLen));//[+文件注释]; - if (isNeedAlign) assert_align(self);//对齐检查; + if (isNeedAlign) + assert_align(self->_curFilePos,isNeedSoAlign?self->_SoPageAlignSize:self->_ZipAlignSize);//对齐检查; return true; } diff --git a/src/patch/Zipper.h b/src/patch/Zipper.h index 0481373..9538b0b 100644 --- a/src/patch/Zipper.h +++ b/src/patch/Zipper.h @@ -113,9 +113,11 @@ bool UnZipper_isHaveApkV2orV3SignTag_in_ApkV1SignFile(UnZipper* self); //found t bool UnZipper_file_isApkV1_or_jarSign(const UnZipper* self,int fileIndex); // file is in ApkV1Sign Dir bool UnZipper_file_isApkV1Sign(const UnZipper* self,int fileIndex); // file is ApkV1Sign File bool UnZipper_file_isReCompressedByApkV2Sign(const UnZipper* self,int fileIndex); + ZipFilePos_t UnZipper_fileEntry_offset_unsafe(const UnZipper* self,int fileIndex); int UnZipper_searchFileIndexByName(const UnZipper* self,const char* fileName,int fileNameLen); bool UnZipper_file_is_sameName(const UnZipper* self,int fileIndex,const char* fileName,int fileNameLen); +bool UnZipper_file_is_nameEndWith(const UnZipper* self,int fileIndex,const char* nameSuffix);//file name is end with nameSuffix struct TZipThreadWorks; @@ -147,6 +149,7 @@ typedef struct Zipper{ int _fileEntryMaxCount; int _fileEntryCount; size_t _ZipAlignSize; + size_t _SoPageAlignSize; int _compressLevel; int _compressMemLevel; int _fileHeaderCount; @@ -172,9 +175,9 @@ typedef struct Zipper{ } Zipper; void Zipper_init(Zipper* self); bool Zipper_openFile(Zipper* self,const char* zipFileName,int fileEntryMaxCount, - int ZipAlignSize,int compressLevel,int compressMemLevel); + int ZipAlignSize,int SoPageAlignSize,int compressLevel,int compressMemLevel); bool Zipper_openStream(Zipper* self,const hpatch_TStreamOutput* zipStream,int fileEntryMaxCount, - int ZipAlignSize,int compressLevel,int compressMemLevel); + int ZipAlignSize,int SoPageAlignSize,int compressLevel,int compressMemLevel); bool Zipper_close(Zipper* self); bool Zipper_file_append_copy(Zipper* self,UnZipper* srcZip,int srcFileIndex, bool isAlwaysReCompress=false); From c9204ab6c18d69b4bab93d51dbf0618141c89648 Mon Sep 17 00:00:00 2001 From: sisong Date: Fri, 8 Dec 2023 15:16:51 +0800 Subject: [PATCH 5/9] remove file stamp-cert-sha256; --- src/diff/DiffData.cpp | 5 +++++ src/diff/DiffData.h | 3 ++- src/normalized/normalized.cpp | 22 +++++++++------------- src/patch/Zipper.cpp | 27 +++++++++++++++++++-------- src/patch/Zipper.h | 5 +++-- 5 files changed, 38 insertions(+), 24 deletions(-) diff --git a/src/diff/DiffData.cpp b/src/diff/DiffData.cpp index 2e719bc..49306eb 100644 --- a/src/diff/DiffData.cpp +++ b/src/diff/DiffData.cpp @@ -61,6 +61,10 @@ bool zipFileData_isSame(UnZipper* selfZip,int selfIndex,UnZipper* srcZip,int src } +bool getIsStampCertFile(const UnZipper* self,int fileIndex){ + return UnZipper_file_is_lastNameWith(self,fileIndex,"stamp-cert-sha256",17); +} + bool getZipIsSameWithStream(const hpatch_TStreamInput* oldZipStream, const hpatch_TStreamInput* newZipStream, int newApkFilesRemoved,bool* out_isOldHaveApkV2Sign){ @@ -99,6 +103,7 @@ bool getZipIsSameWithStream(const hpatch_TStreamInput* oldZipStream, for (std::map::iterator it=oldMap.begin();it!=oldMap.end();++it){ int old_i=it->second; check_clear(UnZipper_file_isApkV1Sign(&oldZip,old_i) + ||getIsStampCertFile(&oldZip,old_i) ||(UnZipper_file_uncompressedSize(&oldZip,old_i)==0)); } } diff --git a/src/diff/DiffData.h b/src/diff/DiffData.h index 563f2db..d3a0228 100644 --- a/src/diff/DiffData.h +++ b/src/diff/DiffData.h @@ -56,11 +56,12 @@ bool getCompressedIsNormalizedBy(UnZipper* zip,int zlibCompressLevel, int zlibCompressMemLevel,bool testReCompressedByApkV2Sign=false); //只检查压缩数据是否标准化; size_t getZipAlignSize_unsafe(UnZipper* zip); //只检查未压缩数据的起始位置对齐值,返回对齐值,0表示未对齐; -static inline std::string zipFile_name(UnZipper* self,int fileIndex){ +static inline std::string zipFile_name(const UnZipper* self,int fileIndex){ int nameLen=UnZipper_file_nameLen(self,fileIndex); const char* nameBegin=UnZipper_file_nameBegin(self,fileIndex); return std::string(nameBegin,nameBegin+nameLen); } +bool getIsStampCertFile(const UnZipper* self,int fileIndex); bool getSamePairList(UnZipper* newZip,UnZipper* oldZip, bool newCompressedDataIsNormalized, diff --git a/src/normalized/normalized.cpp b/src/normalized/normalized.cpp index 558979a..4f2853a 100644 --- a/src/normalized/normalized.cpp +++ b/src/normalized/normalized.cpp @@ -50,19 +50,10 @@ struct TFileValue{ static bool inline isEndWith(const std::string& s,const char* sub){ return (s.find(sub)==s.size()-strlen(sub)); } - static bool inline isNameWith(const std::string& s,const char* name){ - if (!isEndWith(s,name)) return false; - size_t nameL=s.size()-strlen(name)-1; - if (s.size()==nameL) return true; - size_t s1=s.size()-nameL-1; - return isDirTag(s[s1]); - } + static bool inline isSignMFFile(const std::string& s){ return (s=="META-INF/MANIFEST.MF")||(s=="META-INF\\MANIFEST.MF"); } - static bool inline isCertFile(const std::string& s){ - return isNameWith(s,"stamp-cert-sha256"); - } struct TCmp{ inline explicit TCmp(int fileCount):_fileCount(fileCount){} size_t _v(const TFileValue& x)const{ @@ -86,14 +77,14 @@ struct TFileValue{ } }; }; + static void getAllFiles(const UnZipper* zip,std::vector& out_files){ int fileCount=UnZipper_fileCount(zip); out_files.resize(fileCount); for (int i=0; i& files if (nL>ny.size()) continue; if (0!=memcmp(nx.c_str(),ny.c_str(),nL)) continue; if (((nL>0)&&TFileValue::isDirTag(nx[nL-1]))|| - ((nLfileIndex=-1; //need remove } } @@ -171,6 +162,11 @@ bool ZipNormalized(const char* srcApk,const char* dstApk,int ZipAlignSize,int co continue; //remove JarSign(ApkV1Sign) when found ApkV2Sign } } + if (getIsStampCertFile(&unzipper,fileIndex)){ + ++apkFilesRemoved; + removedFiles.push_back(files[i].fileName); + continue; //remove stamp cert file + } fileIndexs.push_back(fileIndex); } diff --git a/src/patch/Zipper.cpp b/src/patch/Zipper.cpp index 306ac08..2ad6236 100644 --- a/src/patch/Zipper.cpp +++ b/src/patch/Zipper.cpp @@ -75,6 +75,9 @@ typedef enum TDataDescriptor{ #define kMinFileHeaderSize (46) //struct size +static bool inline _isDirTag(char c){ + return (c=='\\')|(c=='/'); +} //反向找到第一个kENDHEADERMAGIC的位置; static bool _UnZipper_searchEndCentralDirectory(UnZipper* self,ZipFilePos_t* out_endCentralDirectory_pos){ @@ -219,9 +222,18 @@ bool UnZipper_isHaveApkV1Sign(const UnZipper* self){ return false; } -bool UnZipper_file_is_sameName(const UnZipper* self,int fileIndex,const char* fileName,int fileNameLen){ - return (UnZipper_file_nameLen(self,fileIndex)==fileNameLen)&& - (0==memcmp(UnZipper_file_nameBegin(self,fileIndex),fileName,fileNameLen)); +bool UnZipper_file_is_sameName(const UnZipper* self,int fileIndex,const char* pathName,int pathNameLen){ + return (UnZipper_file_nameLen(self,fileIndex)==pathNameLen)&& + (0==memcmp(UnZipper_file_nameBegin(self,fileIndex),pathName,pathNameLen)); +} + +bool UnZipper_file_is_lastNameWith(const UnZipper* self,int fileIndex,const char* lastName,int lastNameLen){ + if (!UnZipper_file_is_nameEndWith(self,fileIndex,lastName,lastNameLen)) return false; + size_t nameL=UnZipper_file_nameLen(self,fileIndex); + if (lastNameLen==nameL) return true; + size_t s1=lastNameLen-nameL-1; + const char* fn=UnZipper_file_nameBegin(self,fileIndex); + return _isDirTag(fn[s1]); } int UnZipper_searchFileIndexByName(const UnZipper* self,const char* fileName,int fileNameLen){ @@ -300,12 +312,11 @@ bool UnZipper_file_isApkV1_or_jarSign(const UnZipper* self,int fileIndex){ ||(0==memcmp(fn,kJarSignPath1,kJarSignPathLen)); } -bool UnZipper_file_is_nameEndWith(const UnZipper* self,int fileIndex,const char* nameSuffix){ - const int nsLen=(int)strlen(nameSuffix); +bool UnZipper_file_is_nameEndWith(const UnZipper* self,int fileIndex,const char* nameSuffix,int nameSuffixLen){ int fnLen=UnZipper_file_nameLen(self,fileIndex); - if (fnLen_curFilePos+headInfoLen,self->_ZipAlignSize); - if (UnZipper_file_is_nameEndWith(srcZip,srcFileIndex,".so")){ + if (UnZipper_file_is_nameEndWith(srcZip,srcFileIndex,".so",3)){ size_t skipSoLen=_getAlignSkipLen(self->_curFilePos+headInfoLen,self->_SoPageAlignSize); if (skipSoLen!=skipLen){ skipLen=skipSoLen; diff --git a/src/patch/Zipper.h b/src/patch/Zipper.h index 9538b0b..18062f2 100644 --- a/src/patch/Zipper.h +++ b/src/patch/Zipper.h @@ -116,8 +116,9 @@ bool UnZipper_file_isReCompressedByApkV2Sign(const UnZipper* self,int fileIndex) ZipFilePos_t UnZipper_fileEntry_offset_unsafe(const UnZipper* self,int fileIndex); int UnZipper_searchFileIndexByName(const UnZipper* self,const char* fileName,int fileNameLen); -bool UnZipper_file_is_sameName(const UnZipper* self,int fileIndex,const char* fileName,int fileNameLen); -bool UnZipper_file_is_nameEndWith(const UnZipper* self,int fileIndex,const char* nameSuffix);//file name is end with nameSuffix +bool UnZipper_file_is_sameName(const UnZipper* self,int fileIndex,const char* pathName,int pathNameLen); +bool UnZipper_file_is_lastNameWith(const UnZipper* self,int fileIndex,const char* lastName,int lastNameLen); +bool UnZipper_file_is_nameEndWith(const UnZipper* self,int fileIndex,const char* nameSuffix,int nameSuffixLen);//file name is end with nameSuffix struct TZipThreadWorks; From 3a913bbede6c072a26c6ae31ad3587c4510e3518 Mon Sep 17 00:00:00 2001 From: sisong Date: Fri, 8 Dec 2023 15:18:28 +0800 Subject: [PATCH 6/9] update version; --- README.md | 2 +- src/patch/patch_types.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 981da76..7a987b5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # [ApkDiffPatch] -[![release](https://img.shields.io/badge/release-v1.6.2-blue.svg)](https://github.com/sisong/ApkDiffPatch/releases) +[![release](https://img.shields.io/badge/release-v1.6.3-blue.svg)](https://github.com/sisong/ApkDiffPatch/releases) [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/ApkDiffPatch/blob/master/LICENSE) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/ApkDiffPatch/pulls) [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/ApkDiffPatch?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/ApkDiffPatch/issues) diff --git a/src/patch/patch_types.h b/src/patch/patch_types.h index c22f3fb..5585238 100644 --- a/src/patch/patch_types.h +++ b/src/patch/patch_types.h @@ -33,7 +33,7 @@ #define APKDIFFPATCH_VERSION_MAJOR 1 #define APKDIFFPATCH_VERSION_MINOR 6 -#define APKDIFFPATCH_VERSION_RELEASE 2 +#define APKDIFFPATCH_VERSION_RELEASE 3 #define _APKDIFFPATCH_VERSION APKDIFFPATCH_VERSION_MAJOR.APKDIFFPATCH_VERSION_MINOR.APKDIFFPATCH_VERSION_RELEASE #define _APKDIFFPATCH_QUOTE(str) #str From 4b10ed11a12476fef7ab9cab53f1d6c1bc18129f Mon Sep 17 00:00:00 2001 From: sisong Date: Fri, 8 Dec 2023 15:46:42 +0800 Subject: [PATCH 7/9] recode for fix CI; --- src/apk_normalized.cpp | 1 + src/diff/DiffData.cpp | 7 +------ src/diff/DiffData.h | 1 - src/normalized/normalized.cpp | 2 +- src/patch/Patcher.cpp | 2 +- src/patch/Zipper.cpp | 3 ++- src/patch/Zipper.h | 3 +++ 7 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/apk_normalized.cpp b/src/apk_normalized.cpp index 11e1062..d8ceb44 100644 --- a/src/apk_normalized.cpp +++ b/src/apk_normalized.cpp @@ -60,6 +60,7 @@ static void printUsage(){ " -as-alignSize\n" " set align size for uncompressed file in zip for optimize app run speed,\n" " 1 <= alignSize <= 4k, recommended 4,8, DEFAULT -as-8.\n" + " NOTE: if -ap-1, must 4096%%alignSize==0;\n" " -ap-isPageAlignSoFile\n" " if found uncompressed .so file in the zip, need align it to 4k page?\n" " -ap-0 not page-align uncompressed .so files;\n" diff --git a/src/diff/DiffData.cpp b/src/diff/DiffData.cpp index 49306eb..c598d69 100644 --- a/src/diff/DiffData.cpp +++ b/src/diff/DiffData.cpp @@ -60,11 +60,6 @@ bool zipFileData_isSame(UnZipper* selfZip,int selfIndex,UnZipper* srcZip,int src return 0==memcmp(buf.data(),buf.data()+selfFileSize,selfFileSize); } - -bool getIsStampCertFile(const UnZipper* self,int fileIndex){ - return UnZipper_file_is_lastNameWith(self,fileIndex,"stamp-cert-sha256",17); -} - bool getZipIsSameWithStream(const hpatch_TStreamInput* oldZipStream, const hpatch_TStreamInput* newZipStream, int newApkFilesRemoved,bool* out_isOldHaveApkV2Sign){ @@ -103,7 +98,7 @@ bool getZipIsSameWithStream(const hpatch_TStreamInput* oldZipStream, for (std::map::iterator it=oldMap.begin();it!=oldMap.end();++it){ int old_i=it->second; check_clear(UnZipper_file_isApkV1Sign(&oldZip,old_i) - ||getIsStampCertFile(&oldZip,old_i) + ||UnZipper_file_isStampCertFile(&oldZip,old_i) ||(UnZipper_file_uncompressedSize(&oldZip,old_i)==0)); } } diff --git a/src/diff/DiffData.h b/src/diff/DiffData.h index d3a0228..69a2fd9 100644 --- a/src/diff/DiffData.h +++ b/src/diff/DiffData.h @@ -61,7 +61,6 @@ static inline std::string zipFile_name(const UnZipper* self,int fileIndex){ const char* nameBegin=UnZipper_file_nameBegin(self,fileIndex); return std::string(nameBegin,nameBegin+nameLen); } -bool getIsStampCertFile(const UnZipper* self,int fileIndex); bool getSamePairList(UnZipper* newZip,UnZipper* oldZip, bool newCompressedDataIsNormalized, diff --git a/src/normalized/normalized.cpp b/src/normalized/normalized.cpp index 4f2853a..2f5bbb8 100644 --- a/src/normalized/normalized.cpp +++ b/src/normalized/normalized.cpp @@ -162,7 +162,7 @@ bool ZipNormalized(const char* srcApk,const char* dstApk,int ZipAlignSize,int co continue; //remove JarSign(ApkV1Sign) when found ApkV2Sign } } - if (getIsStampCertFile(&unzipper,fileIndex)){ + if (UnZipper_file_isStampCertFile(&unzipper,fileIndex)){ ++apkFilesRemoved; removedFiles.push_back(files[i].fileName); continue; //remove stamp cert file diff --git a/src/patch/Patcher.cpp b/src/patch/Patcher.cpp index b43215b..13e24c4 100644 --- a/src/patch/Patcher.cpp +++ b/src/patch/Patcher.cpp @@ -154,7 +154,7 @@ TPatchResult VirtualZipPatchWithStream(const hpatch_TStreamInput* oldZipStream,c check(oldStream.stream->streamSize==diffInfo.oldDataSize,PATCH_OLDDATA_ERROR); check(Zipper_openStream(&out_newZip,outNewZipStream,(int)zipDiffData.newZipFileCount, - (int)zipDiffData.newZipAlignSize,(int)zipDiffData.newCompressLevel, + (int)zipDiffData.newZipAlignSize,(int)zipDiffData.newZipAlignSize,(int)zipDiffData.newCompressLevel, (int)zipDiffData.newCompressMemLevel),PATCH_OPENWRITE_ERROR); check(NewStream_open(&newStream,&out_newZip,&oldZip, (size_t)diffInfo.newDataSize, zipDiffData.newZipIsDataNormalized!=0, diff --git a/src/patch/Zipper.cpp b/src/patch/Zipper.cpp index 2ad6236..c928aae 100644 --- a/src/patch/Zipper.cpp +++ b/src/patch/Zipper.cpp @@ -1020,7 +1020,8 @@ static bool _write_fileHeaderInfo(Zipper* self,int fileIndex,UnZipper* srcZip,in if (isNeedAlign){ size_t headInfoLen=30+fileNameLen+extraFieldLen; size_t skipLen=_getAlignSkipLen(self->_curFilePos+headInfoLen,self->_ZipAlignSize); - if (UnZipper_file_is_nameEndWith(srcZip,srcFileIndex,".so",3)){ + if ((self->_SoPageAlignSize!=self->_ZipAlignSize)&&UnZipper_file_is_nameEndWith(srcZip,srcFileIndex,".so",3)){ + check(0==(self->_SoPageAlignSize%self->_ZipAlignSize)); size_t skipSoLen=_getAlignSkipLen(self->_curFilePos+headInfoLen,self->_SoPageAlignSize); if (skipSoLen!=skipLen){ skipLen=skipSoLen; diff --git a/src/patch/Zipper.h b/src/patch/Zipper.h index 18062f2..e740744 100644 --- a/src/patch/Zipper.h +++ b/src/patch/Zipper.h @@ -120,6 +120,9 @@ bool UnZipper_file_is_sameName(const UnZipper* self,int fileIndex,const char* pa bool UnZipper_file_is_lastNameWith(const UnZipper* self,int fileIndex,const char* lastName,int lastNameLen); bool UnZipper_file_is_nameEndWith(const UnZipper* self,int fileIndex,const char* nameSuffix,int nameSuffixLen);//file name is end with nameSuffix +static inline bool UnZipper_file_isStampCertFile(const UnZipper* self,int fileIndex){ + return UnZipper_file_is_lastNameWith(self,fileIndex,"stamp-cert-sha256",17); +} struct TZipThreadWorks; struct TZipThreadWork; From d02f936bb48817aee22f6462007d1f517de43185 Mon Sep 17 00:00:00 2001 From: sisong Date: Sat, 9 Dec 2023 08:07:46 +0800 Subject: [PATCH 8/9] recode support uncompressed .so file in the zip align to 4k page; --- builds/vc/ZipDiff.vcxproj | 8 ++--- builds/vc/ZipPatch.vcxproj | 8 ++--- src/diff/DiffData.cpp | 8 ++--- src/diff/Differ.cpp | 3 +- src/normalized/normalized.cpp | 14 +++++--- src/patch/Patcher.cpp | 2 +- src/patch/Zipper.cpp | 66 ++++++++++++++++++++++++++--------- src/patch/Zipper.h | 7 ++-- src/zip_diff.cpp | 6 ++-- 9 files changed, 80 insertions(+), 42 deletions(-) diff --git a/builds/vc/ZipDiff.vcxproj b/builds/vc/ZipDiff.vcxproj index beae942..68da0c1 100644 --- a/builds/vc/ZipDiff.vcxproj +++ b/builds/vc/ZipDiff.vcxproj @@ -134,7 +134,7 @@ Level3 Disabled - WIN32;_DEBUG;%(PreprocessorDefinitions) + WIN32;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) Default /wd4819 %(AdditionalOptions) @@ -149,7 +149,7 @@ Level3 Disabled - WIN32;_DEBUG;%(PreprocessorDefinitions) + WIN32;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) Default /wd4819 %(AdditionalOptions) @@ -163,7 +163,7 @@ Level3 - WIN32;NDEBUG;%(PreprocessorDefinitions) + WIN32;_CRT_SECURE_NO_WARNINGS;NDEBUG;%(PreprocessorDefinitions) Default Speed true @@ -184,7 +184,7 @@ Level3 - WIN32;NDEBUG;%(PreprocessorDefinitions) + WIN32;_CRT_SECURE_NO_WARNINGS;NDEBUG;%(PreprocessorDefinitions) Default Speed true diff --git a/builds/vc/ZipPatch.vcxproj b/builds/vc/ZipPatch.vcxproj index 7f3023e..1404103 100644 --- a/builds/vc/ZipPatch.vcxproj +++ b/builds/vc/ZipPatch.vcxproj @@ -119,7 +119,7 @@ Level3 Disabled - WIN32;_DEBUG;%(PreprocessorDefinitions) + WIN32;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) Default /wd4819 %(AdditionalOptions) @@ -134,7 +134,7 @@ Level3 Disabled - WIN32;_DEBUG;%(PreprocessorDefinitions) + WIN32;_CRT_SECURE_NO_WARNINGS;_DEBUG;%(PreprocessorDefinitions) Default /wd4819 %(AdditionalOptions) @@ -148,7 +148,7 @@ Level3 - WIN32;NDEBUG;%(PreprocessorDefinitions) + WIN32;_CRT_SECURE_NO_WARNINGS;NDEBUG;%(PreprocessorDefinitions) Default Speed true @@ -169,7 +169,7 @@ Level3 - WIN32;NDEBUG;%(PreprocessorDefinitions) + WIN32;_CRT_SECURE_NO_WARNINGS;NDEBUG;%(PreprocessorDefinitions) Default Speed true diff --git a/src/diff/DiffData.cpp b/src/diff/DiffData.cpp index c598d69..399840a 100644 --- a/src/diff/DiffData.cpp +++ b/src/diff/DiffData.cpp @@ -203,7 +203,7 @@ static bool _isAligned(const std::vector& offsetList,ZipFilePos_t size_t getZipAlignSize_unsafe(UnZipper* zip){ //note: 该函数对没有Normalized的zip允许获取AlignSize失败; int fileCount=UnZipper_fileCount(zip); - ZipFilePos_t maxSkipLen=0; + //ZipFilePos_t maxSkipLen=0; ZipFilePos_t minOffset=1024*4; //set search max AlignSize std::vector offsetList; for (int i=0; imaxSkipLen) maxSkipLen=skipLen; + //if (skipLen>maxSkipLen) maxSkipLen=skipLen; ZipFilePos_t offset=UnZipper_fileData_offset(zip,i); if (offset=maxSkipLen+1; --align) { + for (ZipFilePos_t align=minOffset; align>1; --align) { if (_isAligned(offsetList,align)) return align; } - return 0; + return 1; } bool getSamePairList(UnZipper* newZip,UnZipper* oldZip, diff --git a/src/diff/Differ.cpp b/src/diff/Differ.cpp index c304e57..6dd909d 100644 --- a/src/diff/Differ.cpp +++ b/src/diff/Differ.cpp @@ -144,11 +144,10 @@ bool ZipDiffWithStream(const hpatch_TStreamInput* oldZipStream,const hpatch_TStr newZipAlignSize=getZipAlignSize_unsafe(&newZip); if (UnZipper_isHaveApkV2Sign(&newZip)){//precondition (+checkZipIsSame() to complete) - newZip._isDataNormalized=(newZipAlignSize>0)&(newZip._dataDescriptorCount==0); + newZip._isDataNormalized=(newZipAlignSize>0)&&(newZip._dataDescriptorCount==0); }else{ newZip._isDataNormalized=true; } - newZipAlignSize=(newZipAlignSize>0)?newZipAlignSize:kDefaultZipAlignSize; if (newZip._isDataNormalized && UnZipper_isHaveApkV2Sign(&newZip)){ newCompressedDataIsNormalized=getCompressedIsNormalized(&newZip,&newZipNormalized_compressLevel, &newZipNormalized_compressMemLevel); diff --git a/src/normalized/normalized.cpp b/src/normalized/normalized.cpp index 2f5bbb8..9ed533f 100644 --- a/src/normalized/normalized.cpp +++ b/src/normalized/normalized.cpp @@ -36,8 +36,6 @@ if (!(value)){ printf(#value" ERROR!\n"); \ result=false; if (!_isInClear){ goto clear; } } } -#define kSoPageAlignSize (1024*4) - struct TFileValue{ std::string fileName; int fileIndex; @@ -134,6 +132,7 @@ bool ZipNormalized(const char* srcApk,const char* dstApk,int ZipAlignSize,int co int jarSignFileCount=0; std::vector fileIndexs; std::vector removedFiles; + std::vector _compressedEmptyFiles; //only for WARNING UnZipper unzipper; Zipper zipper; UnZipper_init(&unzipper); @@ -141,7 +140,7 @@ bool ZipNormalized(const char* srcApk,const char* dstApk,int ZipAlignSize,int co check(UnZipper_openFile(&unzipper,srcApk)); fileCount=UnZipper_fileCount(&unzipper); - check(Zipper_openFile(&zipper,dstApk,fileCount,ZipAlignSize,isPageAlignSoFile?kSoPageAlignSize:ZipAlignSize, + check(Zipper_openFile(&zipper,dstApk,fileCount,ZipAlignSize,isPageAlignSoFile, compressLevel,kDefaultZlibCompressMemLevel)); isHaveApkV2Sign=UnZipper_isHaveApkV2Sign(&unzipper); isHaveApkV3Sign=UnZipper_isHaveApkV3Sign(&unzipper); @@ -186,8 +185,8 @@ bool ZipNormalized(const char* srcApk,const char* dstApk,int ZipAlignSize,int co check(Zipper_file_append_set_new_isCompress(&zipper,false)); printf("NOTE: \"%s\" is a compressed empty file, change to uncompressed!\n",fileName.c_str()); }else{ - printf("WARNING: \"%s\" is a compressed empty file, can't patch by old(version0){ if (isHaveApkV2Sign){ printf("WARNING: src removed JarSign(ApkV1Sign) (%d file, need re sign)\n",jarSignFileCount); @@ -221,6 +223,8 @@ bool ZipNormalized(const char* srcApk,const char* dstApk,int ZipAlignSize,int co :"WARNING: src removed ApkV2Sign data (%d Byte, need re sign)\n", (int)UnZipper_ApkV2SignSize(&unzipper)); } + if (zipper._normalizeSoPageAlignCount>0) + printf("WARNING: this is new diffFile format, it can't patch by old(versionstreamSize==diffInfo.oldDataSize,PATCH_OLDDATA_ERROR); check(Zipper_openStream(&out_newZip,outNewZipStream,(int)zipDiffData.newZipFileCount, - (int)zipDiffData.newZipAlignSize,(int)zipDiffData.newZipAlignSize,(int)zipDiffData.newCompressLevel, + (int)zipDiffData.newZipAlignSize,false,(int)zipDiffData.newCompressLevel, (int)zipDiffData.newCompressMemLevel),PATCH_OPENWRITE_ERROR); check(NewStream_open(&newStream,&out_newZip,&oldZip, (size_t)diffInfo.newDataSize, zipDiffData.newZipIsDataNormalized!=0, diff --git a/src/patch/Zipper.cpp b/src/patch/Zipper.cpp index c928aae..053977d 100644 --- a/src/patch/Zipper.cpp +++ b/src/patch/Zipper.cpp @@ -35,6 +35,8 @@ #include "../../HDiffPatch/compress_plugin_demo.h" #include "../../HDiffPatch/decompress_plugin_demo.h" +#define kSoPageAlignSize (1024*4) + static const TCompressPlugin_zlib zipCompatibleCompressPlugin={ {_zlib_compressType,_default_maxCompressedSize,_default_setParallelThreadNumber,_zlib_compress}, 9,MAX_MEM_LEVEL,-MAX_WBITS,hpatch_FALSE}; @@ -191,7 +193,7 @@ inline static int32_t _centralDirectory_size(const UnZipper* self){ return readUInt32(self->_endCentralDirectory+12); } -inline static const TByte* fileHeaderBuf(const UnZipper* self,int fileIndex){ +inline static TByte* fileHeaderBuf(const UnZipper* self,int fileIndex){ return self->_centralDirectory+self->_fileHeaderOffsets[fileIndex]; } @@ -524,6 +526,18 @@ bool UnZipper_updateVirtualVCE(UnZipper* self,bool isDataNormalized,size_t zipCE return true; } + +inline static unsigned char* _at_file_generalPurposeBitFlag(const UnZipper* self,int fileIndex){ + return fileHeaderBuf(self,fileIndex)+8; +} +//NOTE: used bit pos 7 in generalPurposeBitFlag for saving isPageAlignSoFile tag +inline static bool _file_getIsPageAlignSoFile(const UnZipper* self,int fileIndex){ + return 0!=((*_at_file_generalPurposeBitFlag(self,fileIndex))&(1<<7)); +} +inline static void _file_setIsPageAlignSoFile(const UnZipper* self,int fileIndex){ + (*_at_file_generalPurposeBitFlag(self,fileIndex))|=(1<<7); +} + inline static const unsigned char* _at_file_compressType(const UnZipper* self,int fileIndex){ return fileHeaderBuf(self,fileIndex)+10; } @@ -863,17 +877,15 @@ bool Zipper_close(Zipper* self){ #define isUsedMT(self) (self->_threadNum>1) bool Zipper_openStream(Zipper* self,const hpatch_TStreamOutput* zipStream,int fileEntryMaxCount, - int ZipAlignSize,int SoPageAlignSize,int compressLevel,int compressMemLevel){ + int ZipAlignSize,bool isNormalizeSoPageAlign,int compressLevel,int compressMemLevel){ #if (_IS_NEED_FIXED_ZLIB_VERSION) check(0==strcmp(kNormalizedZlibVersion,zlibVersion()));//fixed zlib version #endif assert(ZipAlignSize>0); - assert(SoPageAlignSize>0); if (ZipAlignSize<=0) ZipAlignSize=1; - if (SoPageAlignSize<=0) SoPageAlignSize=1; checkCompressSet(compressLevel,compressMemLevel); self->_ZipAlignSize=ZipAlignSize; - self->_SoPageAlignSize=SoPageAlignSize; + self->_isNormalizeSoPageAlign=isNormalizeSoPageAlign; self->_compressLevel=compressLevel; self->_compressMemLevel=compressMemLevel; @@ -922,12 +934,12 @@ void Zipper_by_multi_thread(Zipper* self,int threadNum){ } bool Zipper_openFile(Zipper* self,const char* zipFileName,int fileEntryMaxCount, - int ZipAlignSize,int SoPageAlignSize,int compressLevel,int compressMemLevel){ + int ZipAlignSize,bool isNormalizeSoPageAlign,int compressLevel,int compressMemLevel){ assert(self->_fileStream.m_file==0); check(hpatch_TFileStreamOutput_open(&self->_fileStream,zipFileName,(hpatch_StreamPos_t)(-1))); hpatch_TFileStreamOutput_setRandomOut(&self->_fileStream,hpatch_TRUE); bool result=Zipper_openStream(self,&self->_fileStream.base,fileEntryMaxCount, - ZipAlignSize,SoPageAlignSize,compressLevel,compressMemLevel); + ZipAlignSize,isNormalizeSoPageAlign,compressLevel,compressMemLevel); if (!result){ hpatch_TFileStreamOutput_close(&self->_fileStream); self->_fileStream.m_file=0; @@ -1016,21 +1028,41 @@ static bool _write_fileHeaderInfo(Zipper* self,int fileIndex,UnZipper* srcZip,in uint16_t extraFieldLen=readUInt16(headBuf+30); uint16_t extraFieldLen_for_align=extraFieldLen; bool isNeedAlign=(!isFullInfo)&&(!isCompressed); //dir or 0 size file need align too, same AndroidSDK#zipalign - bool isNeedSoAlign=false; + size_t curAlignSize=self->_ZipAlignSize; if (isNeedAlign){ size_t headInfoLen=30+fileNameLen+extraFieldLen; - size_t skipLen=_getAlignSkipLen(self->_curFilePos+headInfoLen,self->_ZipAlignSize); - if ((self->_SoPageAlignSize!=self->_ZipAlignSize)&&UnZipper_file_is_nameEndWith(srcZip,srcFileIndex,".so",3)){ - check(0==(self->_SoPageAlignSize%self->_ZipAlignSize)); - size_t skipSoLen=_getAlignSkipLen(self->_curFilePos+headInfoLen,self->_SoPageAlignSize); - if (skipSoLen!=skipLen){ - skipLen=skipSoLen; + size_t skipLen=_getAlignSkipLen(self->_curFilePos+headInfoLen,curAlignSize); + if ((kSoPageAlignSize!=curAlignSize)&&UnZipper_file_is_nameEndWith(srcZip,srcFileIndex,".so",3)){ + bool tag_sPageAlignSoFile=_file_getIsPageAlignSoFile(srcZip,srcFileIndex); + bool isNeedSoAlign=false; + if (self->_isNormalizeSoPageAlign){ //in Normalize + if (tag_sPageAlignSoFile){//error + LOG_ERR("ERROR: Normalize unsupport this zip file, required bit pos in general purpose bit flag is already in use.\n"); + return false; + } + if (0!=(kSoPageAlignSize%curAlignSize)){//error + LOG_ERR("ERROR: Normalize unsupport this AlignSize %d\n",(int)curAlignSize); + return false; + } + isNeedSoAlign=true; + }else if (tag_sPageAlignSoFile){ //in patch isNeedSoAlign=true; } + if (isNeedSoAlign){ + curAlignSize=kSoPageAlignSize; + size_t skipSoLen=_getAlignSkipLen(self->_curFilePos+headInfoLen,curAlignSize); + if (skipSoLen!=skipLen){ + skipLen=skipSoLen; + if (self->_isNormalizeSoPageAlign){ + _file_setIsPageAlignSoFile(srcZip,srcFileIndex); + ++self->_normalizeSoPageAlignCount; + } + } + } } + if (skipLen>0){ - if ((isNeedSoAlign) //为了保持兼容旧的patch,用extraField来对齐; - ||(fileIndex==0)){//只能扩展第一个entry的extraField来对齐; + if (fileIndex==0){//只能扩展第一个entry的extraField来对齐; extraFieldLen_for_align+=(uint16_t)skipLen; {// print WARNING char fileName[hpatch_kPathMaxSize]; @@ -1077,7 +1109,7 @@ static bool _write_fileHeaderInfo(Zipper* self,int fileIndex,UnZipper* srcZip,in if (fileCommentLen>0) check(_write(self,headBuf+46+fileNameLen+extraFieldLen,fileCommentLen));//[+文件注释]; if (isNeedAlign) - assert_align(self->_curFilePos,isNeedSoAlign?self->_SoPageAlignSize:self->_ZipAlignSize);//对齐检查; + assert_align(self->_curFilePos,curAlignSize);//对齐检查; return true; } diff --git a/src/patch/Zipper.h b/src/patch/Zipper.h index e740744..46c803a 100644 --- a/src/patch/Zipper.h +++ b/src/patch/Zipper.h @@ -153,7 +153,8 @@ typedef struct Zipper{ int _fileEntryMaxCount; int _fileEntryCount; size_t _ZipAlignSize; - size_t _SoPageAlignSize; + bool _isNormalizeSoPageAlign; + int _normalizeSoPageAlignCount; int _compressLevel; int _compressMemLevel; int _fileHeaderCount; @@ -179,9 +180,9 @@ typedef struct Zipper{ } Zipper; void Zipper_init(Zipper* self); bool Zipper_openFile(Zipper* self,const char* zipFileName,int fileEntryMaxCount, - int ZipAlignSize,int SoPageAlignSize,int compressLevel,int compressMemLevel); + int ZipAlignSize,bool isNormalizeSoPageAlign,int compressLevel,int compressMemLevel); bool Zipper_openStream(Zipper* self,const hpatch_TStreamOutput* zipStream,int fileEntryMaxCount, - int ZipAlignSize,int SoPageAlignSize,int compressLevel,int compressMemLevel); + int ZipAlignSize,bool isNormalizeSoPageAlign,int compressLevel,int compressMemLevel); bool Zipper_close(Zipper* self); bool Zipper_file_append_copy(Zipper* self,UnZipper* srcZip,int srcFileIndex, bool isAlwaysReCompress=false); diff --git a/src/zip_diff.cpp b/src/zip_diff.cpp index 3c97354..529c01d 100644 --- a/src/zip_diff.cpp +++ b/src/zip_diff.cpp @@ -69,7 +69,9 @@ static void printUsage(){ # if (_IS_USED_MULTITHREAD) " support run by multi-thread parallel, fast!\n" # endif - " WARNING: code not compatible with it compressed by -c-lzma!\n" + " NOTE: code not compatible with it compressed by -c-lzma!\n" + " WARNING: this is new diffFile format, it can't patch by old(version=0, DEFAULT -m-3.\n" @@ -77,7 +79,7 @@ static void printUsage(){ " create single compressed diffData(DEFAULT closed), only need one decompress buffer\n" " when patch, and support step by step patching when step by step downloading!\n" " stepSize>=" _HDIFFPATCH_EXPAND_AND_QUOTE(hpatch_kStreamCacheSize) ", DEFAULT -SD-2m, recommended 64k,512k,8m etc...\n" - " WARNING: this is new diffFile format, must update patcher for support it.\n" + " WARNING: this is new diffFile format, it can't patch by old(version Date: Sat, 9 Dec 2023 08:23:56 +0800 Subject: [PATCH 9/9] update version; --- README.md | 2 +- src/patch/patch_types.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7a987b5..dbea5ac 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # [ApkDiffPatch] -[![release](https://img.shields.io/badge/release-v1.6.3-blue.svg)](https://github.com/sisong/ApkDiffPatch/releases) +[![release](https://img.shields.io/badge/release-v1.7.0-blue.svg)](https://github.com/sisong/ApkDiffPatch/releases) [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/ApkDiffPatch/blob/master/LICENSE) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/ApkDiffPatch/pulls) [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/ApkDiffPatch?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/ApkDiffPatch/issues) diff --git a/src/patch/patch_types.h b/src/patch/patch_types.h index 5585238..51b0b3a 100644 --- a/src/patch/patch_types.h +++ b/src/patch/patch_types.h @@ -32,8 +32,8 @@ #include //uint32_t uint16_t #define APKDIFFPATCH_VERSION_MAJOR 1 -#define APKDIFFPATCH_VERSION_MINOR 6 -#define APKDIFFPATCH_VERSION_RELEASE 3 +#define APKDIFFPATCH_VERSION_MINOR 7 +#define APKDIFFPATCH_VERSION_RELEASE 0 #define _APKDIFFPATCH_VERSION APKDIFFPATCH_VERSION_MAJOR.APKDIFFPATCH_VERSION_MINOR.APKDIFFPATCH_VERSION_RELEASE #define _APKDIFFPATCH_QUOTE(str) #str